{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/peremartra/Large-Language-Model-Notebooks-Course/blob/main/P1-NL2SQL/nl2sql_prompt_SQLCoder.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div align=\"center\">\n",
        "<h1><a href=\"https://github.com/peremartra/Large-Language-Model-Notebooks-Course\">Learn by Doing LLM Projects</a></h1>\n",
        "    <h3>Understand And Apply Large Language Models</h3>\n",
        "    <h2>Natural Language to SQL with OpenSource LLMs</h2>\n",
        "    <h3>Create a prompt to generate SQL with defog SQLCoder 7B Model</h3>\n",
        "    by <b>Pere Martra</b>\n",
        "</div>\n",
        "\n",
        "<br>\n",
        "\n",
        "<div align=\"center\">\n",
        "    &nbsp;\n",
        "    <a target=\"_blank\" href=\"https://www.linkedin.com/in/pere-martra/\"><img src=\"https://img.shields.io/badge/style--5eba00.svg?label=LinkedIn&logo=linkedin&style=social\"></a>\n",
        "    \n",
        "</div>\n",
        "\n",
        "<br>\n",
        "<hr>"
      ],
      "metadata": {
        "id": "p9zxlAS6w6EB"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1yFnrczEWDMm"
      },
      "source": [
        "Starting with a [prompt specially designed for OpenAI](https://github.com/peremartra/Large-Language-Model-Notebooks-Course/blob/main/P1-NL2SQL/nl2sql_enhaced_ohaio.ipynb), we adapt it for a model like SQLCoder 7B.\n",
        "\n",
        "This model have  Mistral as a base and it has been fine-tuned to excel in SQL code generation."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MGJAHqIsXjhK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3039a867-3a04-4e0f-9e7a-8310522042ef"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.4/302.4 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.8/119.8 MB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m245.2/245.2 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.0/542.0 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.4/102.4 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m18.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.2/401.2 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "# Install the necesary libraries:\n",
        "!pip -q install accelerate bitsandbytes\n",
        "!pip -q install trl"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8VNBGXh9V952",
        "outputId": "def25141-6673-451f-98be-ed277c2a81aa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for peft (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "#Install the lastest versions of peft & transformers library recommended\n",
        "#if you want to work with the most recent models\n",
        "!pip install -q git+https://github.com/huggingface/peft.git\n",
        "!pip install -q git+https://github.com/huggingface/transformers.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U1mZ27vbXBCE"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rotsX3i-XLJS"
      },
      "outputs": [],
      "source": [
        "model_name = \"defog/sqlcoder-7b\""
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We need to create the Quantization configuration to load the Model.\n",
        "\n",
        "It is a large model and I want it to fit in a 16GB GPU, I'm going to use a 4 bits quantization.\n",
        "\n",
        "If you want to learn more about quantization, refer to this article: [QLoRA: Training a Large Language Model on a 16GB GPU.](https://medium.com/towards-artificial-intelligence/qlora-training-a-large-language-model-on-a-16gb-gpu-00ea965667c1)\n",
        "\n",
        "You can try to use this model in a 8 bit quantizations and check in you see any improvements in the results."
      ],
      "metadata": {
        "id": "2fHZsPTa5yun"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Oio2Fm7GXa6K"
      },
      "outputs": [],
      "source": [
        "bnb_config = BitsAndBytesConfig(\n",
        "  load_in_4bit=True,\n",
        "  bnb_4bit_use_double_quant=True,\n",
        "  bnb_4bit_quant_type=\"nf4\",\n",
        "  bnb_4bit_compute_dtype=torch.bfloat16\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "To load the model I pass to the AutoModelForCasualLM teh quantization configurations, and HuggingFace take care of all the hard work."
      ],
      "metadata": {
        "id": "915oTn7T6fvW"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197,
          "referenced_widgets": [
            "59df1d23ec8e41f8a9b91d304fd2884b",
            "3ee4cdb9b5bf468da3e11455c1c3c989",
            "d9cfa66b4aef410a88941fd773787b06",
            "0e0c2b55bf114450a38101407a49d854",
            "e25b73c7613a4a87b5c48c1921dc054a",
            "56a923f61dc44cdab7c189abe39eb367",
            "4d7024ea8c8a4319b515b534f05716df",
            "1e8c08fa71794422ab091dd4efa362d2",
            "e0329b259f49400bb24e23ce79d846dd",
            "15aae3e3f100464f8ff1063548d0d373",
            "16ac11ad434348bcb98dc6c6414c838b",
            "5c1ecc88b1b04cca90493ca3d6169c46",
            "a6f9917e3384406a8d9443180b2f626b",
            "ecf4610e12fe44929587b05740456c91",
            "6e0d2fb2545346609fede13bd15cc414",
            "bb166fce2de04f619a1bfb7281476823",
            "c963010a34ed495d8b8ac29511ed3ba8",
            "9657c4f4a9bd4623a84e794135ca4ead",
            "c8353e8e7cc24e6a8555dcc7bc5338aa",
            "eaa78df20dfc4a7ca612afe44deea07a",
            "1663caddd3c1482e8bcbe1630b138a56",
            "bdb69adc88934228a8f08bb17ceb27c4",
            "510a9df1f52044aaa9c27bcdece9ee5e",
            "df998d89d637404b8c390c0123e4101d",
            "9a61d60a1523488d85e465919be32d66",
            "30dc3824bf9a43f084c6558e61c51260",
            "cdf9fcb95fb0417aa18d906d0ca5d29f",
            "991d8508073041fea86c359984f6b477",
            "2588016978bf48c2977955dff0b276b8",
            "23b6503dafb540d9b7516de261ae9083",
            "ad15ba49769f45a896db783222c56d73",
            "4780d80e8e104891a5b3b7f9f0464626",
            "bf11b8f687f440fcb95c2f026986b9d6",
            "c1e051f955c046eca958cd35c8ee16b6",
            "9a6ab55f93dc4242be873bac827b9a8b",
            "00026ecfa39c4649886331393acb9886",
            "69f0d0c0bb3e44eabc4491a5eb6acdd8",
            "429899740d4b4fbd82f41c9ee7ae7f77",
            "cd5b6c0c5f9e4b45bd03f6df6b69e40d",
            "29ef16348bf14e2a9f0db8e3821a568d",
            "818440b08095400b9acf67d8f8faff4a",
            "966bf5c2d9564165832f42461fc332ea",
            "9156e33d079d4afc9c20132742447eac",
            "21807cd192244cc2b966f3746f0f99a1",
            "4e64b133e720437d95d8c2c2c275aa57",
            "c1d73d9f88514ac58a23c8d32b92dc64",
            "15aeb306ea5d4601b3a4534e1b451682"
          ]
        },
        "id": "Q8-VpiCBXcst",
        "outputId": "891e342a-6a6f-4ec9-9c80-0db762327113"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "59df1d23ec8e41f8a9b91d304fd2884b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "config.json:   0%|          | 0.00/619 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5c1ecc88b1b04cca90493ca3d6169c46",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "pytorch_model.bin.index.json:   0%|          | 0.00/23.9k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "510a9df1f52044aaa9c27bcdece9ee5e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c1e051f955c046eca958cd35c8ee16b6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "pytorch_model-00001-of-00002.bin:   0%|          | 0.00/9.94G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4e64b133e720437d95d8c2c2c275aa57",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "pytorch_model-00002-of-00002.bin:   0%|          | 0.00/4.54G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c1d73d9f88514ac58a23c8d32b92dc64",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "15aeb306ea5d4601b3a4534e1b451682",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/116 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "foundation_model = AutoModelForCausalLM.from_pretrained(model_name,\n",
        "                    quantization_config=bnb_config,\n",
        "                    device_map='auto',\n",
        "                    use_cache = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "4186a842c6d64112bf752bb5782f8cdc",
            "93847fdd31dd4f24b845b13c8e3548c5",
            "967d74ee71434069a0456522b5816908",
            "f5f9fdaaa5d342cea33cb24140023397"
          ]
        },
        "id": "E2Oz6Sm8X5Qo",
        "outputId": "70d3235c-3c00-452d-befe-c6a4d56386b7"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4186a842c6d64112bf752bb5782f8cdc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/915 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93847fdd31dd4f24b845b13c8e3548c5",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer.model:   0%|          | 0.00/493k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "967d74ee71434069a0456522b5816908",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.80M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f5f9fdaaa5d342cea33cb24140023397",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/72.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "eos_token_id = tokenizer.convert_tokens_to_ids([\"```\"])[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This function wraps the call to *model.generate*"
      ],
      "metadata": {
        "id": "NV-Qk6327Z30"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l4vEDNRGtvdn"
      },
      "outputs": [],
      "source": [
        "#this function returns the outputs from the model received, and inputs.\n",
        "def get_outputs(model, inputs, max_new_tokens=400):\n",
        "    outputs = model.generate(\n",
        "        input_ids=inputs[\"input_ids\"],\n",
        "        attention_mask=inputs[\"attention_mask\"],\n",
        "        num_return_sequences=1,\n",
        "        eos_token_id=eos_token_id,\n",
        "        pad_token_id=eos_token_id,\n",
        "        max_new_tokens=max_new_tokens,\n",
        "        do_sample=False,\n",
        "        num_beams=2\n",
        "    )\n",
        "    return outputs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y88PVExjKCvQ"
      },
      "source": [
        "#Prompt without Shots.\n",
        "In this first PROMPT we are going to give Instructions to the model and pass the structure of the Database.\n",
        "\n",
        "The instructions are significantly different from those we are passing to GPT-3.5-Turbo. This model is really well fine-tuned, but it is smaller than GPT-3.5.\n",
        "\n",
        "We need to be more clear with the instructions, as it does not have the same capacity to understand our orders as GPT-3.5."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U8K4XWDQJH_p"
      },
      "outputs": [],
      "source": [
        "sp_nl2sql = \"\"\"\n",
        "[INST]\n",
        "<<SYS>>\n",
        "Your task is convert a question into a SQL query, given a SQL database schema.\n",
        "Adhere to these rules:\n",
        "- **Deliberately go through the question and database schema word by word to appropriately answer the question\n",
        "\n",
        "    ### Input\n",
        "    Generate a SQL query that answers the question below.\n",
        "    This query will run on a database whose schema is represented in this string:\n",
        "\n",
        "    create table employees(\n",
        "        ID_Usr INT primary key,-- Unique Id for employee\n",
        "        name VARCHAR -- Name of employee\n",
        "        );\n",
        "\n",
        "    create table salary(\n",
        "        ID_Usr INT,-- Unique Id for employee\n",
        "        year DATE, -- Date\n",
        "        salary FLOAT, --Salary of employee\n",
        "        foreign key (ID_Usr) references employees(ID_Usr) -- Join Employees with salary\n",
        "        );\n",
        "\n",
        "    create table studies(\n",
        "        ID_study INT, -- Unique ID study\n",
        "        ID_Usr INT, -- ID employee\n",
        "        educational_level INT,  -- 5=phd, 4=Master, 3=Bachelor\n",
        "        Institution VARCHAR, --Name of instituon where eployee studied\n",
        "        Years DATE, -- Date acomplishement stdy\n",
        "        Speciality VARCHAR, -- Speciality of studies\n",
        "        primary key (ID_study, ID_Usr), --Primary Key ID_Usr + ID_Study\n",
        "        foreign key(ID_Usr) references employees (ID_Usr)\n",
        "        );\n",
        "\n",
        "    ### Response\n",
        "    Based on your instructions, here is the SQL query I have generated to answer the question\n",
        "    `{question}`:\n",
        "    ```sql3\n",
        "    \"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tIgEZpmSlG0I",
        "outputId": "bfd1e879-a865-4c82-da3a-bf05a054c633"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "    ### Instructions:\n",
            "Your task is convert a question into a SQL query, given a SQL database schema.\n",
            "Adhere to these rules:\n",
            "- **Deliberately go through the question and database schema word by word** to appropriately answer the question\n",
            "\n",
            "    ### Input\n",
            "    Generate a SQL query that answers the question below.\n",
            "    This query will run on a database whose schema is represented in this string:\n",
            "\n",
            "    create table employees(\n",
            "        ID_Usr INT primary key,-- Unique Id for employee\n",
            "        name VARCHAR -- Name of employee\n",
            "        );\n",
            "\n",
            "    create table salary(\n",
            "        ID_Usr INT,-- Unique Id for employee\n",
            "        year DATE, -- Date\n",
            "        salary FLOAT, --Salary of employee\n",
            "        foreign key (ID_Usr) references employees(ID_Usr) -- Join Employees with salary\n",
            "        );\n",
            "\n",
            "    create table studies(\n",
            "        ID_study INT, -- Unique ID study\n",
            "        ID_Usr INT, -- ID employee\n",
            "        educational_level INT,  -- 5=phd, 4=Master, 3=Bachelor\n",
            "        Institution VARCHAR, --Name of instituon where eployee studied\n",
            "        Years DATE, -- Date acomplishement stdy\n",
            "        Speciality VARCHAR, -- Speciality of studies\n",
            "        primary key (ID_study, ID_Usr), --Primary Key ID_Usr + ID_Study\n",
            "        foreign key(ID_Usr) references employees (ID_Usr)\n",
            "        );\n",
            "\n",
            "    ### Response\n",
            "    Based on your instructions, here is the SQL query I have generated to answer the question\n",
            "    `Return The name of the best paid employee`:\n",
            "    ```sql3\n",
            "    \n"
          ]
        }
      ],
      "source": [
        "sp_nl2sql = sp_nl2sql.format(question=\"Return The name of the best paid employee\")\n",
        "print(sp_nl2sql)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Mp3TDRQtltX9"
      },
      "outputs": [],
      "source": [
        "input_sentences = tokenizer(sp_nl2sql, return_tensors=\"pt\").to('cuda')\n",
        "response = get_outputs(foundation_model, input_sentences, max_new_tokens=400)\n",
        "SQL = tokenizer.batch_decode(response, skip_special_tokens=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hWouSh-7rpCk"
      },
      "outputs": [],
      "source": [
        "#Empty the cache in order to do more calls without problems.\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g0V5XuDp0sCi",
        "outputId": "56a72fc1-24d6-4742-908f-b8eb3a431904"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "SELECT employees.name, MAX(salary.salary) AS max_salary FROM employees JOIN salary ON employees.ID_Usr = salary.ID_Usr GROUP BY employees.name ORDER BY max_salary DESC LIMIT 1;\n"
          ]
        }
      ],
      "source": [
        "print(SQL[0].split(\"```sql3\")[-1].split(\"```\")[0].split(\";\")[0].strip() + \";\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The SQL Order is correct."
      ],
      "metadata": {
        "id": "VKrjztcK-zDm"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d0hogn4fKP1f"
      },
      "source": [
        "#Prompt with shots OpenAI Style.\n",
        "In this second prompt we are going to add some Shots with samples to see if our SQL style affects the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qNPXi3twuN7-"
      },
      "outputs": [],
      "source": [
        "sp_nl2sql2 = \"\"\"\n",
        "[INST]\n",
        "<<SYS>>\n",
        "Your task is convert a question into a SQL query, given a SQL database schema.\n",
        "Adhere to these rules:\n",
        "- **Deliberately go through the question and database schema word by word** to appropriately answer the question\n",
        "- **Use the samples SQL In the ### Samples section to clearn more about teh Databases structure\n",
        "\n",
        "\n",
        "    ### Input\n",
        "    Generate a SQL query that answers the question below.\n",
        "    This query will run on a database whose schema is represented in this string:\n",
        "\n",
        "    create table employees(\n",
        "        ID_Usr INT primary key,-- Unique Id for employee\n",
        "        name VARCHAR -- Name of employee\n",
        "        );\n",
        "\n",
        "    create table salary(\n",
        "        ID_Usr INT,-- Unique Id for employee\n",
        "        year DATE, -- Date\n",
        "        salary FLOAT, --Salary of employee\n",
        "        foreign key (ID_Usr) references employees(ID_Usr) -- Join Employees with salary\n",
        "        );\n",
        "\n",
        "    create table studies(\n",
        "        ID_study INT, -- Unique ID study\n",
        "        ID_Usr INT, -- ID employee\n",
        "        educational_level INT,  -- 5=phd, 4=Master, 3=Bachelor\n",
        "        Institution VARCHAR, --Name of instituon where eployee studied\n",
        "        Years DATE, -- Date acomplishement stdy\n",
        "        Speciality VARCHAR, -- Speciality of studies\n",
        "        primary key (ID_study, ID_Usr), --Primary Key ID_Usr + ID_Study\n",
        "        foreign key(ID_Usr) references employees (ID_Usr)\n",
        "        );\n",
        "<</SYS>>\n",
        "\n",
        "\n",
        "    ### Response\n",
        "    Question: `How Many employes we have with a salary bigger than 50000?`:\n",
        "    SELECT COUNT(*) AS total_employees\n",
        "    FROM employees e\n",
        "    INNER JOIN salary s ON e.ID_Usr = s.ID_Usr\n",
        "    WHERE s.salary > 50000;\n",
        "\n",
        "    Question: `Return the names of the three people who have had the highest salary increase in the last three years.`\n",
        "    SELECT e.name\n",
        "    FROM employees e\n",
        "    JOIN salary s ON e.ID_usr = s.ID_usr\n",
        "    WHERE s.year >= DATE_SUB(CURDATE(), INTERVAL 3 YEAR)\n",
        "    GROUP BY e.name\n",
        "    ORDER BY (MAX(s.salary) - MIN(s.salary)) DESC\n",
        "    LIMIT 3;\n",
        "\n",
        "    `{question}`:\n",
        "    ```sql3\n",
        "    [/INST]\n",
        "    \"\"\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7vy338SDLr5u",
        "outputId": "d4b646fe-707f-4885-9a32-aeae4a992c3e"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "    ### Instructions:\n",
            "Your task is convert a question into a SQL query, given a SQL database schema.\n",
            "Adhere to these rules:\n",
            "- **Deliberately go through the question and database schema word by word** to appropriately answer the question\n",
            "- **Use the samples SQL In the ### Samples section to clearn more about teh Databases structure\n",
            "\n",
            "\n",
            "    ### Input\n",
            "    Generate a SQL query that answers the question below.\n",
            "    This query will run on a database whose schema is represented in this string:\n",
            "\n",
            "    create table employees(\n",
            "        ID_Usr INT primary key,-- Unique Id for employee\n",
            "        name VARCHAR -- Name of employee\n",
            "        );\n",
            "\n",
            "    create table salary(\n",
            "        ID_Usr INT,-- Unique Id for employee\n",
            "        year DATE, -- Date\n",
            "        salary FLOAT, --Salary of employee\n",
            "        foreign key (ID_Usr) references employees(ID_Usr) -- Join Employees with salary\n",
            "        );\n",
            "\n",
            "    create table studies(\n",
            "        ID_study INT, -- Unique ID study\n",
            "        ID_Usr INT, -- ID employee\n",
            "        educational_level INT,  -- 5=phd, 4=Master, 3=Bachelor\n",
            "        Institution VARCHAR, --Name of instituon where eployee studied\n",
            "        Years DATE, -- Date acomplishement stdy\n",
            "        Speciality VARCHAR, -- Speciality of studies\n",
            "        primary key (ID_study, ID_Usr), --Primary Key ID_Usr + ID_Study\n",
            "        foreign key(ID_Usr) references employees (ID_Usr)\n",
            "        );\n",
            "\n",
            "\n",
            "\n",
            "    ### Response\n",
            "    Question: `How Many employes we have with a salary bigger than 50000?`:\n",
            "    SELECT COUNT(*) AS total_employees\n",
            "    FROM employees e\n",
            "    INNER JOIN salary s ON e.ID_Usr = s.ID_Usr\n",
            "    WHERE s.salary > 50000;\n",
            "\n",
            "    Question: `Return the names of the three people who have had the highest salary increase in the last three years.`\n",
            "    SELECT e.name\n",
            "    FROM employees e\n",
            "    JOIN salary s ON e.ID_usr = s.ID_usr\n",
            "    WHERE s.year >= DATE_SUB(CURDATE(), INTERVAL 3 YEAR)\n",
            "    GROUP BY e.name\n",
            "    ORDER BY (MAX(s.salary) - MIN(s.salary)) DESC\n",
            "    LIMIT 3;\n",
            "\n",
            "    `Return The name of the best paid employee`:\n",
            "    ```sql3\n",
            "    \n"
          ]
        }
      ],
      "source": [
        "sp_nl2sql2 = sp_nl2sql2.format(question=\"Return The name of the best paid employee\")\n",
        "(print(sp_nl2sql2))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cvk9hJfhLwQJ"
      },
      "outputs": [],
      "source": [
        "input_sentences = tokenizer(sp_nl2sql2, return_tensors=\"pt\").to('cuda')\n",
        "response = get_outputs(foundation_model, input_sentences, max_new_tokens=400)\n",
        "SQL = tokenizer.batch_decode(response, skip_special_tokens=True)\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EQOyi8ZOL1mr",
        "outputId": "98a428c1-5f48-4f37-8327-fa4bd9151ace"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "SELECT e.name\n",
            "    FROM employees e\n",
            "    JOIN salary s ON e.ID_Usr = s.ID_usr\n",
            "    WHERE s.salary = (SELECT MAX(salary) FROM salary);\n"
          ]
        }
      ],
      "source": [
        "print(SQL[0].split(\"```sql3\")[-1].split(\"```\")[0].split(\";\")[0].strip() + \";\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "I tested this prompt with different Beams. With one beam the model can't produce a correct answer.\n",
        "\n",
        "Beams 1:\n",
        "SELECT e.name\n",
        "    FROM employees e\n",
        "    JOIN studies s ON e.ID_Usr = s.ID_Usr\n",
        "    WHERE s.year >= DATE_SUB(CURDATE(), INTERVAL 1 YEAR)\n",
        "    GROUP BY e.name\n",
        "    ORDER BY COUNT(s.ID_study) DESC\n",
        "    LIMIT 1;\n",
        "\n",
        "Beams 3:\n",
        "SELECT e.name\n",
        "    FROM employees e\n",
        "    JOIN salary s ON e.ID_Usr = s.ID_usr\n",
        "    WHERE s.salary = (SELECT MAX(salary) FROM salary);\n",
        "\n",
        "Beams 5:\n",
        "SELECT e.name\n",
        "    FROM employees e\n",
        "    JOIN salary s ON e.ID_Usr = s.ID_usr\n",
        "    WHERE s.salary = (SELECT MAX(salary) FROM salary);"
      ],
      "metadata": {
        "id": "bir44lxcQ_YL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The Order is really different from the one obtained with the first prompt.\n",
        "\n",
        "The first difference is the format. But The SQL is realy more simple, at least it is my sensation."
      ],
      "metadata": {
        "id": "buwuhLBw_MAV"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-6S6gI_fNtFk"
      },
      "source": [
        "#Prompt with Shots in Sample Style.\n",
        "\n",
        "In this prompt, we will place the examples in a separate section, and in the instructions, we will instruct the model to pay attention to them in order to generate the SQL commands."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oHSF1l3xNscJ"
      },
      "outputs": [],
      "source": [
        "sp_nl2sql3b = \"\"\"\n",
        "    ### Instructions:\n",
        "Your task is convert a question into a SQL query, given a SQL database schema.\n",
        "Adhere to these rules:\n",
        "- **Deliberately go through the question and database schema word by word** to appropriately answer the question\n",
        "- **Use the samples SQL In the ### Samples section to learn more about the Databases structure\n",
        "\n",
        "\n",
        "    ### Input\n",
        "    Generate a SQL query that answers the question below.\n",
        "    This query will run on a database whose schema is represented in this string:\n",
        "\n",
        "    create table employees(\n",
        "        ID_Usr INT primary key,-- Unique Id for employee\n",
        "        name VARCHAR -- Name of employee\n",
        "        );\n",
        "\n",
        "    create table salary(\n",
        "        ID_Usr INT,-- Unique Id for employee\n",
        "        year DATE, -- Date\n",
        "        salary FLOAT, --Salary of employee\n",
        "        foreign key (ID_Usr) references employees(ID_Usr) -- Join Employees with salary\n",
        "        );\n",
        "\n",
        "    create table studies(\n",
        "        ID_study INT, -- Unique ID study\n",
        "        ID_Usr INT, -- ID employee\n",
        "        educational_level INT,  -- 5=phd, 4=Master, 3=Bachelor\n",
        "        Institution VARCHAR, --Name of instituon where eployee studied\n",
        "        Years DATE, -- Date acomplishement stdy\n",
        "        Speciality VARCHAR, -- Speciality of studies\n",
        "        primary key (ID_study, ID_Usr), --Primary Key ID_Usr + ID_Study\n",
        "        foreign key(ID_Usr) references employees (ID_Usr)\n",
        "        );\n",
        "\n",
        "    ### Samples\n",
        "    Question: `How Many employes we have with a salary bigger than 50000?`:\n",
        "    SELECT COUNT(*) AS total_employees\n",
        "    FROM employees e\n",
        "    INNER JOIN salary s ON e.ID_Usr = s.ID_Usr\n",
        "    WHERE s.salary > 50000;\n",
        "\n",
        "    Question: `Return the names of the three people who have had the highest salary increase in the last three years.`\n",
        "    SELECT e.name\n",
        "    FROM employees e\n",
        "    JOIN salary s ON e.ID_usr = s.ID_usr\n",
        "    WHERE s.year >= DATE_SUB(CURDATE(), INTERVAL 3 YEAR)\n",
        "    GROUP BY e.name\n",
        "    ORDER BY (MAX(s.salary) - MIN(s.salary)) DESC\n",
        "    LIMIT 3;\n",
        "\n",
        "    ### Response\n",
        "    Based on your instructions, here is the SQL query I have generated to answer the question\n",
        "    `{question}`:\n",
        "    ```sql3\n",
        "    \"\"\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DCfbmvDyOS-u",
        "outputId": "b5a732f2-0d7b-488a-ec9f-f4ceb23b18bf"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "    ### Instructions:\n",
            "Your task is convert a question into a SQL query, given a SQL database schema.\n",
            "Adhere to these rules:\n",
            "- **Deliberately go through the question and database schema word by word** to appropriately answer the question\n",
            "- **Use the samples SQL In the ### Samples section to learn more about the Databases structure\n",
            "\n",
            "\n",
            "    ### Input\n",
            "    Generate a SQL query that answers the question below.\n",
            "    This query will run on a database whose schema is represented in this string:\n",
            "\n",
            "    create table employees(\n",
            "        ID_Usr INT primary key,-- Unique Id for employee\n",
            "        name VARCHAR -- Name of employee\n",
            "        );\n",
            "\n",
            "    create table salary(\n",
            "        ID_Usr INT,-- Unique Id for employee\n",
            "        year DATE, -- Date\n",
            "        salary FLOAT, --Salary of employee\n",
            "        foreign key (ID_Usr) references employees(ID_Usr) -- Join Employees with salary\n",
            "        );\n",
            "\n",
            "    create table studies(\n",
            "        ID_study INT, -- Unique ID study\n",
            "        ID_Usr INT, -- ID employee\n",
            "        educational_level INT,  -- 5=phd, 4=Master, 3=Bachelor\n",
            "        Institution VARCHAR, --Name of instituon where eployee studied\n",
            "        Years DATE, -- Date acomplishement stdy\n",
            "        Speciality VARCHAR, -- Speciality of studies\n",
            "        primary key (ID_study, ID_Usr), --Primary Key ID_Usr + ID_Study\n",
            "        foreign key(ID_Usr) references employees (ID_Usr)\n",
            "        );\n",
            "\n",
            "    ### Samples\n",
            "    Question: `How Many employes we have with a salary bigger than 50000?`:\n",
            "    SELECT COUNT(*) AS total_employees\n",
            "    FROM employees e\n",
            "    INNER JOIN salary s ON e.ID_Usr = s.ID_Usr\n",
            "    WHERE s.salary > 50000;\n",
            "\n",
            "    Question: `Return the names of the three people who have had the highest salary increase in the last three years.`\n",
            "    SELECT e.name\n",
            "    FROM employees e\n",
            "    JOIN salary s ON e.ID_usr = s.ID_usr\n",
            "    WHERE s.year >= DATE_SUB(CURDATE(), INTERVAL 3 YEAR)\n",
            "    GROUP BY e.name\n",
            "    ORDER BY (MAX(s.salary) - MIN(s.salary)) DESC\n",
            "    LIMIT 3;\n",
            "\n",
            "    ### Response\n",
            "    Based on your instructions, here is the SQL query I have generated to answer the question\n",
            "    `Return The name of the best paid employee`:\n",
            "    ```sql3\n",
            "    \n"
          ]
        }
      ],
      "source": [
        "sp_nl2sql3 = sp_nl2sql3b.format(question=\"Return The name of the best paid employee\")\n",
        "print (sp_nl2sql3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VxsIf4KBOWCk"
      },
      "outputs": [],
      "source": [
        "input_sentences = tokenizer(sp_nl2sql3, return_tensors=\"pt\").to('cuda')\n",
        "response = get_outputs(foundation_model, input_sentences, max_new_tokens=400)\n",
        "SQL = tokenizer.batch_decode(response, skip_special_tokens=True)\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_sE5_a8jOaJ7",
        "outputId": "03c5abd5-844c-4ebe-e387-03767384bfb6"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "SELECT e.name\n",
            "    FROM employees e\n",
            "    JOIN salary s ON e.ID_Usr = s.ID_Usr\n",
            "    WHERE s.salary = (SELECT MAX(salary) FROM salary);\n"
          ]
        }
      ],
      "source": [
        "print(SQL[0].split(\"```sql3\")[-1].split(\"```\")[0].split(\";\")[0].strip() + \";\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l5OgzBzKNHzn"
      },
      "source": [
        "#Now the question in spanish.\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sp_nl2sql3 = sp_nl2sql3b.format(question=\"Devuelveme el nombre del empleado mejor pagado\")\n",
        "print (sp_nl2sql3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EQuJNMjNFU6a",
        "outputId": "cf0ce2f3-afbb-4c9d-d72b-ac31594c009b"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "    ### Instructions:\n",
            "Your task is convert a question into a SQL query, given a SQL database schema.\n",
            "Adhere to these rules:\n",
            "- **Deliberately go through the question and database schema word by word** to appropriately answer the question\n",
            "- **Use the samples SQL In the ### Samples section to learn more about the Databases structure\n",
            "\n",
            "\n",
            "    ### Input\n",
            "    Generate a SQL query that answers the question below.\n",
            "    This query will run on a database whose schema is represented in this string:\n",
            "\n",
            "    create table employees(\n",
            "        ID_Usr INT primary key,-- Unique Id for employee\n",
            "        name VARCHAR -- Name of employee\n",
            "        );\n",
            "\n",
            "    create table salary(\n",
            "        ID_Usr INT,-- Unique Id for employee\n",
            "        year DATE, -- Date\n",
            "        salary FLOAT, --Salary of employee\n",
            "        foreign key (ID_Usr) references employees(ID_Usr) -- Join Employees with salary\n",
            "        );\n",
            "\n",
            "    create table studies(\n",
            "        ID_study INT, -- Unique ID study\n",
            "        ID_Usr INT, -- ID employee\n",
            "        educational_level INT,  -- 5=phd, 4=Master, 3=Bachelor\n",
            "        Institution VARCHAR, --Name of instituon where eployee studied\n",
            "        Years DATE, -- Date acomplishement stdy\n",
            "        Speciality VARCHAR, -- Speciality of studies\n",
            "        primary key (ID_study, ID_Usr), --Primary Key ID_Usr + ID_Study\n",
            "        foreign key(ID_Usr) references employees (ID_Usr)\n",
            "        );\n",
            "\n",
            "    ### Samples\n",
            "    Question: `How Many employes we have with a salary bigger than 50000?`:\n",
            "    SELECT COUNT(*) AS total_employees\n",
            "    FROM employees e\n",
            "    INNER JOIN salary s ON e.ID_Usr = s.ID_Usr\n",
            "    WHERE s.salary > 50000;\n",
            "\n",
            "    Question: `Return the names of the three people who have had the highest salary increase in the last three years.`\n",
            "    SELECT e.name\n",
            "    FROM employees e\n",
            "    JOIN salary s ON e.ID_usr = s.ID_usr\n",
            "    WHERE s.year >= DATE_SUB(CURDATE(), INTERVAL 3 YEAR)\n",
            "    GROUP BY e.name\n",
            "    ORDER BY (MAX(s.salary) - MIN(s.salary)) DESC\n",
            "    LIMIT 3;\n",
            "\n",
            "    ### Response\n",
            "    Based on your instructions, here is the SQL query I have generated to answer the question\n",
            "    `Devuelveme el nombre del empleado mejor pagado`:\n",
            "    ```sql3\n",
            "    \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input_sentences = tokenizer(sp_nl2sql3, return_tensors=\"pt\").to('cuda')\n",
        "response = get_outputs(foundation_model, input_sentences, max_new_tokens=400)\n",
        "SQL = tokenizer.batch_decode(response, skip_special_tokens=True)\n",
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "rBGokR4RFfrK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(SQL[0].split(\"```sql3\")[-1].split(\"```\")[0].split(\";\")[0].strip() + \";\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OEoQgcoUHXXJ",
        "outputId": "05c542fc-c497-4abd-fe67-549dd555ac74"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "SELECT e.name\n",
            "    FROM employees e\n",
            "    INNER JOIN salary s ON e.ID_Usr = s.ID_Usr\n",
            "    WHERE s.salary = (SELECT MAX(salary) FROM salary);\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The generated SQL command is the same regardless of where we have placed the examples."
      ],
      "metadata": {
        "id": "pHZn6nGCH2RS"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Conclusions.\n",
        "\n",
        "Let's see the three SQL's together.\n",
        "\n",
        "* SELECT employees.name, MAX(salary.salary) AS max_salary FROM employees JOIN salary ON employees.ID_Usr = salary.ID_Usr GROUP BY employees.name ORDER BY max_salary DESC NULLS LAST LIMIT 1;\n",
        "\n",
        "* SELECT e.name\n",
        "    FROM employees e\n",
        "    JOIN salary s ON e.ID_Usr = s.ID_usr\n",
        "    WHERE s.salary = (SELECT MAX(salary) FROM salary);\n",
        "\n",
        "* SELECT e.name\n",
        "    FROM employees e\n",
        "    JOIN salary s ON e.ID_Usr = s.ID_usr\n",
        "    WHERE s.salary = (SELECT MAX(salary) FROM salary);\n",
        "\n",
        "* Spanish Question: SELECT e.name\n",
        "     FROM employees e\n",
        "     JOIN salary s ON e.ID_Usr = s.ID_Usr\n",
        "     WHERE s.salary = (SELECT MAX(salary) FROM salary)\n",
        "     GROUP BY e.name\n",
        "     ORDER BY COUNT(studies.ID_study) DESC\n",
        "     LIMIT 1;\n",
        "\n",
        "\n",
        "**The model has demonstrated that it is highly efficient in crafting SQL.** Additionally, it pays a lot of attention, perhaps too much, to the examples we provide. Clearly, these examples should be crafted by one of the best SQL programmers we have access to, though their use may not be essential.\n",
        "\n",
        "On the other hand, although the model is clearly very proficient in SQL generation, during the creation of the notebook, I have encountered several issues because the commands need to be extremely clear. It doesn't handle typos well (which should not exist).\n",
        "\n",
        "It appears to have some issues when it receives commands in Spanish. I assume this problem would be present in any language other than English. Therefore, since it's a tool that could be used by non-technical personnel, this should be considered in environments where English is not the primary language."
      ],
      "metadata": {
        "id": "yrHGZjraCYSR"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "P45ukTZyHdFE"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "gpuType": "L4",
      "authorship_tag": "ABX9TyP8T7aStJXCqN8eCzu1Yu0q",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "59df1d23ec8e41f8a9b91d304fd2884b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3ee4cdb9b5bf468da3e11455c1c3c989",
              "IPY_MODEL_d9cfa66b4aef410a88941fd773787b06",
              "IPY_MODEL_0e0c2b55bf114450a38101407a49d854"
            ],
            "layout": "IPY_MODEL_e25b73c7613a4a87b5c48c1921dc054a"
          }
        },
        "3ee4cdb9b5bf468da3e11455c1c3c989": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_56a923f61dc44cdab7c189abe39eb367",
            "placeholder": "​",
            "style": "IPY_MODEL_4d7024ea8c8a4319b515b534f05716df",
            "value": "config.json: 100%"
          }
        },
        "d9cfa66b4aef410a88941fd773787b06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1e8c08fa71794422ab091dd4efa362d2",
            "max": 619,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e0329b259f49400bb24e23ce79d846dd",
            "value": 619
          }
        },
        "0e0c2b55bf114450a38101407a49d854": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_15aae3e3f100464f8ff1063548d0d373",
            "placeholder": "​",
            "style": "IPY_MODEL_16ac11ad434348bcb98dc6c6414c838b",
            "value": " 619/619 [00:00&lt;00:00, 55.1kB/s]"
          }
        },
        "e25b73c7613a4a87b5c48c1921dc054a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "56a923f61dc44cdab7c189abe39eb367": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d7024ea8c8a4319b515b534f05716df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1e8c08fa71794422ab091dd4efa362d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0329b259f49400bb24e23ce79d846dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "15aae3e3f100464f8ff1063548d0d373": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16ac11ad434348bcb98dc6c6414c838b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5c1ecc88b1b04cca90493ca3d6169c46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a6f9917e3384406a8d9443180b2f626b",
              "IPY_MODEL_ecf4610e12fe44929587b05740456c91",
              "IPY_MODEL_6e0d2fb2545346609fede13bd15cc414"
            ],
            "layout": "IPY_MODEL_bb166fce2de04f619a1bfb7281476823"
          }
        },
        "a6f9917e3384406a8d9443180b2f626b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c963010a34ed495d8b8ac29511ed3ba8",
            "placeholder": "​",
            "style": "IPY_MODEL_9657c4f4a9bd4623a84e794135ca4ead",
            "value": "pytorch_model.bin.index.json: 100%"
          }
        },
        "ecf4610e12fe44929587b05740456c91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c8353e8e7cc24e6a8555dcc7bc5338aa",
            "max": 23950,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_eaa78df20dfc4a7ca612afe44deea07a",
            "value": 23950
          }
        },
        "6e0d2fb2545346609fede13bd15cc414": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1663caddd3c1482e8bcbe1630b138a56",
            "placeholder": "​",
            "style": "IPY_MODEL_bdb69adc88934228a8f08bb17ceb27c4",
            "value": " 23.9k/23.9k [00:00&lt;00:00, 2.02MB/s]"
          }
        },
        "bb166fce2de04f619a1bfb7281476823": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c963010a34ed495d8b8ac29511ed3ba8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9657c4f4a9bd4623a84e794135ca4ead": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c8353e8e7cc24e6a8555dcc7bc5338aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eaa78df20dfc4a7ca612afe44deea07a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1663caddd3c1482e8bcbe1630b138a56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bdb69adc88934228a8f08bb17ceb27c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "510a9df1f52044aaa9c27bcdece9ee5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_df998d89d637404b8c390c0123e4101d",
              "IPY_MODEL_9a61d60a1523488d85e465919be32d66",
              "IPY_MODEL_30dc3824bf9a43f084c6558e61c51260"
            ],
            "layout": "IPY_MODEL_cdf9fcb95fb0417aa18d906d0ca5d29f"
          }
        },
        "df998d89d637404b8c390c0123e4101d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_991d8508073041fea86c359984f6b477",
            "placeholder": "​",
            "style": "IPY_MODEL_2588016978bf48c2977955dff0b276b8",
            "value": "Downloading shards:   0%"
          }
        },
        "9a61d60a1523488d85e465919be32d66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_23b6503dafb540d9b7516de261ae9083",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ad15ba49769f45a896db783222c56d73",
            "value": 0
          }
        },
        "30dc3824bf9a43f084c6558e61c51260": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4780d80e8e104891a5b3b7f9f0464626",
            "placeholder": "​",
            "style": "IPY_MODEL_bf11b8f687f440fcb95c2f026986b9d6",
            "value": " 0/2 [00:00&lt;?, ?it/s]"
          }
        },
        "cdf9fcb95fb0417aa18d906d0ca5d29f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "991d8508073041fea86c359984f6b477": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2588016978bf48c2977955dff0b276b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "23b6503dafb540d9b7516de261ae9083": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ad15ba49769f45a896db783222c56d73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4780d80e8e104891a5b3b7f9f0464626": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf11b8f687f440fcb95c2f026986b9d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c1e051f955c046eca958cd35c8ee16b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9a6ab55f93dc4242be873bac827b9a8b",
              "IPY_MODEL_00026ecfa39c4649886331393acb9886",
              "IPY_MODEL_69f0d0c0bb3e44eabc4491a5eb6acdd8"
            ],
            "layout": "IPY_MODEL_429899740d4b4fbd82f41c9ee7ae7f77"
          }
        },
        "9a6ab55f93dc4242be873bac827b9a8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd5b6c0c5f9e4b45bd03f6df6b69e40d",
            "placeholder": "​",
            "style": "IPY_MODEL_29ef16348bf14e2a9f0db8e3821a568d",
            "value": "pytorch_model-00001-of-00002.bin:  81%"
          }
        },
        "00026ecfa39c4649886331393acb9886": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_818440b08095400b9acf67d8f8faff4a",
            "max": 9943030860,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_966bf5c2d9564165832f42461fc332ea",
            "value": 8021606400
          }
        },
        "69f0d0c0bb3e44eabc4491a5eb6acdd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9156e33d079d4afc9c20132742447eac",
            "placeholder": "​",
            "style": "IPY_MODEL_21807cd192244cc2b966f3746f0f99a1",
            "value": " 8.02G/9.94G [02:47&lt;00:33, 57.6MB/s]"
          }
        },
        "429899740d4b4fbd82f41c9ee7ae7f77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd5b6c0c5f9e4b45bd03f6df6b69e40d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "29ef16348bf14e2a9f0db8e3821a568d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "818440b08095400b9acf67d8f8faff4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "966bf5c2d9564165832f42461fc332ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9156e33d079d4afc9c20132742447eac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21807cd192244cc2b966f3746f0f99a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}